"""
@Author: CheyenneLam
@Date: 2020-12-16 10:31:00
@LastEditTime: 2020-12-16 10:31:00
@LastEditors: CheyenneLam
@Description: 各数据源巡检
@FilePath: \Main\main.py
"""

import time
import json
import logging
import traceback
import acm
from interval import Interval

from sql import *
from acm_handle import *
from conf.var_set import EnvChoose, Threshold
from dding_notification import DingTalk

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s %(name)s %(levelname)s %(message)s",
                    datefmt='%Y-%m-%d  %H:%M:%S %a'
                    )


def send_notify(env_dDing, no_privilege_tables, unequal_tables, db_type):
    error_table = ""
    for index, no_privilege in enumerate(no_privilege_tables):
        if error_table == "":
            error_table = "{0}.".format(index + 1) + str(no_privilege) + "\n"
        else:
            error_table = error_table + "{0}.".format(index + 1) + str(no_privilege) + "\n"

    unequal_table = ""
    for unequal_list in unequal_tables:
        if unequal_table == "":
            unequal_table = str(unequal_list[0]) + " 原表:" + str(unequal_list[1]) + "," \
                            + " 同步表:" + str(unequal_list[2]) \
                            + "," + " 相差:" + str(unequal_list[3]) + "。" + "\n"
        else:
            unequal_table = unequal_table + \
                            str(unequal_list[0]) + " 原表:" + str(unequal_list[1]) + "," \
                            + " 同步表:" + str(unequal_list[2]) \
                            + "," + " 相差:" + str(unequal_list[3]) + "。" + "\n"

    if error_table == "":
        error_table = "无\n"
    if unequal_table == "":
        unequal_table = "无\n"

    notify_content = {}
    notify_content["msgtype"] = "markdown"
    notify_content["markdown"] = {}
    notify_content["markdown"]["title"] = "{0}巡检".format(db_type)
    notify_content["markdown"]["text"] = "### {0}巡检 \n #### 以下表查询出错： \n ".format(
        db_type) + error_table + "#### 以下表数据量不一致： \n " + unequal_table

    try:
        ding_talk = DingTalk(env_dDing)
        ding_talk.sendMessage(notify_content)
    except Exception as e:
        logging.error("Send Error: {0}".format(traceback.format_exc()))
        return False
    else:
        return True


def trigger_alarm(source_size, dw_size):
    if source_size in Interval(0, 100):
        trigger_value = Threshold['small']
    elif source_size in Interval(101, 1000):
        trigger_value = Threshold['middle']
    elif source_size in Interval(1001, 10000):
        trigger_value = Threshold['big']
    elif source_size in Interval(10001, 100000):
        trigger_value = Threshold['large']
    elif source_size in Interval(100001, 5000000):
        trigger_value = Threshold['huge']
    else:
        trigger_value = Threshold['extra_large']

    if abs(source_size - dw_size) > trigger_value:
        return True
    else:
        return False


def mongo_check(env_dDing, env_mg, full_mongo_tables):
    try:
        xh_mg_op = MongoDB(env_mg)

        source_size_dict = {}
        for mgTableName, _ in full_mongo_tables:
            dw_size = xh_mg_op.query_collection_count(mgTableName)
            source_size_dict[mgTableName] = dw_size
        logging.warning('mongo查询结果:{0}'.format(source_size_dict))
        xh_mg_op.close()

        mg_size_dict = {}
        no_privilege_table = []
        unequal_table = []
        send_dding = False
        for mgTableName, viewTableName in full_mongo_tables:
            try:
                dataworks_size = dw_conn.query_sql(f"select count(*) from {viewTableName}").values[0][0]
                mg_size_dict[viewTableName] = dataworks_size

                source_size = source_size_dict[mgTableName]
                tri_status = trigger_alarm(source_size, dataworks_size)
                if tri_status:
                    unequal_table.append([viewTableName, source_size, dataworks_size, source_size - dataworks_size])
                    send_dding = True

            except Exception as e:
                logging.error("{0}_ERROR:{1}".format(mgTableName, e))
                no_privilege_table.append([mgTableName, e])

        logging.warning('dataworks同步mongo表查询结果:{0}'.format(mg_size_dict))
        if send_dding:
            send_status = send_notify(env_dDing, no_privilege_table, unequal_table, "Mongo")
            if send_status:
                logging.warning("Send DingTalk Success")
            else:
                logging.warning("Send DingTalk Failed")
    except Exception as e:
        logging.error("Mongo Check ERROR:{0}".format(traceback.format_exc()))
        return False
    else:
        return True


def pg_check(env_dDing, env_pg, full_pg_tables):
    try:
        xh_pg_op = PgDB(env_pg)

        source_size_dict = {}
        for pgTableName, _ in full_pg_tables:
            sql = f"select count(*) from {pgTableName}"
            dw_size = xh_pg_op.query_sql(sql).values[0][0]
            source_size_dict[pgTableName] = dw_size
        logging.warning('pg查询结果:{0}'.format(source_size_dict))
        xh_pg_op.close()

        pg_size_dict = {}
        no_privilege_table = []
        unequal_table = []
        send_dding = False
        for pgTableName, viewTableName in full_pg_tables:
            try:
                start_time = time.time()
                dataworks_size = dw_conn.query_sql(f"select count(*) from {viewTableName}").values[0][0]
                pg_size_dict[viewTableName] = dataworks_size
                end_time = time.time()
                logging.warning('{0}-use_time:{1}'.format(viewTableName, int(end_time - start_time)))

                source_size = source_size_dict[pgTableName]
                tri_status = trigger_alarm(source_size, dataworks_size)
                if tri_status:
                    unequal_table.append([viewTableName, source_size, dataworks_size, source_size - dataworks_size])
                    send_dding = True

            except Exception as e:
                logging.error("{0}_ERROR:{1}".format(pgTableName, e))
                no_privilege_table.append([pgTableName, e])

        logging.warning('dataworks同步pg表查询结果:{0}'.format(pg_size_dict))
        if send_dding:
            send_status = send_notify(env_dDing, no_privilege_table, unequal_table, "Pg")
            if send_status:
                logging.warning("Send DingTalk Success")
            else:
                logging.warning("Send DingTalk Failed")
    except Exception as e:
        logging.error("Pg Check ERROR:{0}".format(traceback.format_exc()))
        return False
    else:
        return True


if __name__ == '__main__':
    # 配置获取
    env_dw, env_pg, env_mg, env_dDing = projectMapping[str(EnvChoose)]

    # acm获取配置信息
    acm_client = acm.ACMClient(ENDPOINT, NAMESPACE, AK, SK)

    # postgresql
    pg_config = {}
    pg_watching = Info('Postgres', 'Octopus')
    pg_content = pg_watching.get_config(acm_client)
    if pg_content is None:
        local_pg_config = get_config(env_pg)
        pg_config['db_name'] = local_pg_config['db_name']
        pg_config['username'] = local_pg_config['username']
        pg_config['password'] = local_pg_config['password']
        pg_config['uri'] = local_pg_config['uri']
        pg_config['port'] = local_pg_config['port']
    else:
        pg_content = json.loads(pg_content)
        pg_config['db_name'] = pg_content['db_name']
        pg_config['username'] = pg_content['username']
        pg_config['password'] = pg_content['password']
        pg_config['uri'] = pg_content['uri'].split(':')[0]
        pg_config['port'] = pg_content['uri'].split(':')[1]
    logging.warning("pg_config: {0}".format(pg_config))

    # mongo
    mongo_config = {}
    mongo_watching = Info('Mongo', 'Octopus')
    mongo_content = mongo_watching.get_config(acm_client)
    if mongo_content is None:
        local_mongo_config = get_config(env_mg)
        mongo_config['db_name'] = local_mongo_config['db_name']
        mongo_config['username'] = local_mongo_config['username']
        mongo_config['password'] = local_mongo_config['password']
        mongo_config['uri'] = local_mongo_config['uri']
        mongo_config['port'] = local_mongo_config['port']
    else:
        mongo_content = json.loads(mongo_content)
        mongo_config['db_name'] = mongo_content['db']
        mongo_config['username'] = mongo_content['username']
        mongo_config['password'] = mongo_content['password']
        mongo_config['uri'] = mongo_content['hostport'][0].split(':')[0]
        mongo_config['port'] = mongo_content['hostport'][0].split(':')[1]
    logging.warning("mongo_config: {0}".format(mongo_config))

    dw_conn = OdpsDB(env_dw)
    table_list_df = DataFrame(dw_conn.o.get_table('system_table_info')).to_pandas()

    full_mongo_tables = table_list_df.query("data_base == 'mongo'")[['data_source_table', 'dw_view_name']].values
    check_status_mongo = mongo_check(env_dDing, mongo_config, full_mongo_tables)
    if check_status_mongo:
        logging.warning("check mongo Success")
    else:
        logging.error("check mongo Failed")

    full_pg_tables = table_list_df.query("data_base == 'pg'")[['data_source_table', 'dw_view_name']].values
    check_status_pg = pg_check(env_dDing, pg_config, full_pg_tables)
    if check_status_pg:
        logging.warning("checkt pg Success")
    else:
        logging.error("check pg Failed")
